from dragon_ai import *


def has_no_chinese(s):
    """
    判断字符串是否完全不含有中文

    参数:
        s: 待检查的字符串

    返回:
        True: 完全不含中文
        False: 含有中文
    """
    # 匹配中文的正则表达式（包括基本汉字和常见中文符号）
    chinese_pattern = re.compile(
        r'[\u4e00-\u9fa5\u3002\uff1b\uff0c\uff1a\u201c\u201d\uff08\uff09\u3001\uff1f\u300a\u300b]')
    # 检查是否有匹配的中文
    return not bool(chinese_pattern.search(s))

def do_check_data(data_list_path, res_output_path):
   dict_list = load_dict_list_from_jsonl(data_list_path)
   # see data type of dict_list
   data_type = {}
   for dict_i in tqdm(dict_list, desc="data_type", total=len(dict_list)):
      extra_dict = dict_i['extra']
      txt = dict_i['txt']
      is_en = has_no_chinese(txt)
      age = extra_dict.get('age', 'unknown')
      gender = extra_dict.get('gender', 'unknown')
      dataset = extra_dict.get('dataset', 'unknown')
      if dataset not in data_type:
         data_type[dataset] = {
            'total': 0,
            'only_age': 0,
            'only_gender': 0,
            'none': 0,
            'both': 0,
            'cn': 0,
            'en': 0
         }
      if age != 'unknown' and gender != 'unknown':
         data_type[dataset]['both'] += 1
      elif age != 'unknown':
         data_type[dataset]['only_age'] += 1
      elif gender != 'unknown':
         data_type[dataset]['only_gender'] += 1
      else:
         data_type[dataset]['none'] += 1
      if is_en:
         data_type[dataset]['en'] += 1
      else:
         data_type[dataset]['cn'] += 1
      data_type[dataset]['total'] += 1
   print_info("data_type:\n", data_type)
   write_dict_to_json(data_type, res_output_path)

if __name__ == '__main__':
   """"""
   data_list_path = "/home/A02_tmpdata3/osum_s2s/gender/data.list"
   res_output_path = "./data_type4gender.json"
   do_check_data(data_list_path, res_output_path)

   data_list_path = "/home/A02_tmpdata3/osum_s2s/gender_kaggle/data.list"
   res_output_path = "./data_type4kaggle_ gender.json"
   do_check_data(data_list_path, res_output_path)

   data_list_path = "/home/A02_tmpdata3/osum_s2s/gender_add_2025_2_4_fix/data.list"
   res_output_path = "./data_type4gender_add_2025_2_4_fix.json"
   do_check_data(data_list_path, res_output_path)

   data_list_path = "/home/A02_tmpdata3/osum_s2s/gender_with_noize_add_2025_2_4/data.list"
   res_output_path = "./data_type4gender_with_noize_add_2025_2_4.json"
   do_check_data(data_list_path, res_output_path)

   data_list_path = "/home/A02_tmpdata3/osum_s2s/age_3000_noize/data.list"
   res_output_path = "./data_type4age_3000_noize.json"
   do_check_data(data_list_path, res_output_path)
